{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th></tr><tr><td>34</td><td>application_1637773802226_0043</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"/hopsworks-api/yarnui/https://hopsworks0.logicalclocks.com:8089/proxy/application_1637773802226_0043/\">Link</a></td><td><a target=\"_blank\" href=\"/hopsworks-api/yarnui/https://hopsworks0.logicalclocks.com:8044/node/containerlogs/container_1637773802226_0043_01_000001/demo_ml_meb10000__meb10000\">Link</a></td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    }
   ],
   "source": [
    "def tensorflow_model_local_absolute(mr, name, version=None, metrics=None, description=None, input_example=None, model_schema=None):\n",
    "    print(\"LOCAL ABSOLUTE MODEL PATH\")\n",
    "    import os\n",
    "    import random\n",
    "    from hsml import model\n",
    "    \n",
    "    model_path = os.getcwd() + \"/model\"\n",
    "    if not os.path.exists(model_path):\n",
    "        os.mkdir(model_path)\n",
    "        f = open(model_path + \"/saved_model.pb\", \"w+\")\n",
    "        f.write(\"model\")\n",
    "        f.close()\n",
    "    \n",
    "    tf_model = mr.tensorflow.create_model(name, metrics=metrics, description=description, input_example=input_example, model_schema=model_schema)\n",
    "    ret = tf_model.save(model_path)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tensorflow_model_local_relative(mr, name, version=None, metrics=None, description=None, input_example=None, model_schema=None):\n",
    "    print(\"LOCAL RELATIVE MODEL PATH\")\n",
    "    import os\n",
    "    import random\n",
    "    from hsml import model\n",
    "    \n",
    "    model_path = os.getcwd() + \"/model\"\n",
    "    if not os.path.exists(model_path):\n",
    "        os.mkdir(model_path)\n",
    "        f = open(model_path + \"/saved_model.pb\", \"w+\")\n",
    "        f.write(\"model\")\n",
    "        f.close()\n",
    "    \n",
    "    tf_model = mr.tensorflow.create_model(name, metrics=metrics, description=description, input_example=input_example, model_schema=model_schema)\n",
    "    ret = tf_model.save('model')\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sklearn_model(mr, name, metrics=None, description=None, input_example=None, model_schema=None):\n",
    "    \n",
    "    print(\"HDFS ABSOLUTE WITH HDFS PREFIX MODEL PATH\")\n",
    "    \n",
    "    import os\n",
    "    import random\n",
    "    from hsml import model\n",
    "    from hops import hdfs\n",
    "    \n",
    "    # Export model from project absolute hdfs path with hdfs prefix\n",
    "    model_path = \"hdfs:///Projects/{}/Resources\".format(hdfs.project_name())\n",
    "\n",
    "    sklearn_model = mr.sklearn.create_model(name, metrics=metrics, description=description, input_example=input_example, model_schema=model_schema)\n",
    "    ret = sklearn_model.save(model_path)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def torch_model(mr, name, metrics=None, description=None, input_example=None, model_schema=None):\n",
    "    \n",
    "    print(\"HDFS ABSOLUTE WITHOUT HDFS PREFIX MODEL PATH\")\n",
    "    \n",
    "    import os\n",
    "    import random\n",
    "    from hsml import model\n",
    "    from hops import hdfs\n",
    "    \n",
    "    # Export model from project absolute hdfs path\n",
    "    model_path = \"/Projects/{}/Resources\".format(hdfs.project_name())\n",
    "\n",
    "    torch_model = mr.torch.create_model(name, metrics=metrics, description=description, input_example=input_example, model_schema=model_schema)\n",
    "    ret = torch_model.save(model_path)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def python_model(mr, name, metrics=None, description=None, input_example=None, model_schema=None):\n",
    "    \n",
    "    print(\"HDFS RELATIVE MODEL PATH\")\n",
    "    \n",
    "    import os\n",
    "    import random\n",
    "    from hsml import model\n",
    "    \n",
    "    # Export model from project relative hdfs path\n",
    "    model_path = \"Resources\"\n",
    "\n",
    "    python_model = mr.python.create_model(name, metrics=metrics, description=description, input_example=input_example, model_schema=model_schema)\n",
    "\n",
    "    ret = python_model.save(model_path)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_tf():\n",
    "    from hsml.schema import Schema\n",
    "    from hsml.model_schema import ModelSchema\n",
    "    import numpy\n",
    "    import pandas\n",
    "    import os\n",
    "    import hsml\n",
    "    from hsml.client.exceptions import RestAPIError\n",
    "    \n",
    "    connection = hsml.connection()\n",
    "    mr = connection.get_model_registry()\n",
    "    \n",
    "    # tf model 1\n",
    "    metrics = {'accuracy': 0.333, 'loss': 0.4}\n",
    "\n",
    "    description = \"A test desc for this model\"\n",
    "\n",
    "    input_data = {'int_column': [1, 2], 'string_column': [\"John\", \"Jamie\"], 'float_column': [0.5, 0.3]}\n",
    "    input_df = pandas.DataFrame(data=input_data)\n",
    "\n",
    "    predictions = numpy.array([1.1, 20.2, 30.3, 40.4, 50.0, 60.6, 70.7, 0.1])\n",
    "\n",
    "    model_schema = ModelSchema(input_schema=Schema(input_df), output_schema=Schema(predictions))\n",
    "\n",
    "    exported_tf_model = tensorflow_model_local_absolute(mr, \"model_tf\", version=1, metrics=metrics, description=description, model_schema=model_schema, input_example=input_df)\n",
    "\n",
    "    # tf model 2\n",
    "    metrics = {'accuracy': 0.95, 'loss': 0.4}\n",
    "\n",
    "    exported_tf_model = tensorflow_model_local_absolute(mr, \"model_tf\", metrics=metrics, description=description, model_schema=model_schema, input_example=input_df)\n",
    "    \n",
    "    # tf model 3\n",
    "    metrics = {'accuracy': 0.90, 'loss': 0.4}\n",
    "\n",
    "    exported_tf_model = tensorflow_model_local_relative(mr, \"model_tf\", version=3, metrics=metrics, description=description, model_schema=model_schema, input_example=input_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_py():\n",
    "    import numpy\n",
    "    import pandas\n",
    "    import os\n",
    "    import hsml\n",
    "    from hsml.client.exceptions import RestAPIError\n",
    "    \n",
    "    connection = hsml.connection()\n",
    "    mr = connection.get_model_registry()\n",
    "    \n",
    "    input_data = {'int_column': [1, 2], 'string_column': [\"John\", \"Jamie\"], 'float_column': [0.5, 0.3]}\n",
    "    input_df = pandas.DataFrame(data=input_data)\n",
    "    \n",
    "    exported_python_model = python_model(mr, \"model_python\", input_example=input_df)\n",
    "    assert isinstance(exported_python_model, hsml.python.model.Model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_sklearn():\n",
    "    import numpy\n",
    "    import pandas\n",
    "    import os\n",
    "    import hsml\n",
    "    from hsml.client.exceptions import RestAPIError\n",
    "    \n",
    "    connection = hsml.connection()\n",
    "    mr = connection.get_model_registry()\n",
    "    \n",
    "    input_data = {'int_column': [1, 2], 'string_column': [\"John\", \"Jamie\"], 'float_column': [0.5, 0.3]}\n",
    "    input_df = pandas.DataFrame(data=input_data)\n",
    "    \n",
    "    exported_sklearn_model = sklearn_model(mr, \"model_sklearn\", input_example=input_df)\n",
    "    assert isinstance(exported_sklearn_model, hsml.sklearn.model.Model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_torch():\n",
    "    import numpy\n",
    "    import pandas\n",
    "    import os\n",
    "    import hsml\n",
    "    from hsml.client.exceptions import RestAPIError\n",
    "    \n",
    "    connection = hsml.connection()\n",
    "    mr = connection.get_model_registry()\n",
    "    \n",
    "    input_data = {'int_column': [1, 2], 'string_column': [\"John\", \"Jamie\"], 'float_column': [0.5, 0.3]}\n",
    "    input_df = pandas.DataFrame(data=input_data)\n",
    "    \n",
    "    exported_torch_model = torch_model(mr, \"model_torch\", input_example=input_df)\n",
    "    assert isinstance(exported_torch_model, hsml.torch.model.Model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Experiment \n",
      "\n",
      "Finished Experiment \n",
      "\n",
      "Finished Experiment \n",
      "\n",
      "Finished Experiment \n",
      "\n",
      "('hdfs://rpc.namenode.service.consul:8020/Projects/demo_ml_meb10000/Experiments/application_1637773802226_0043_4', {'metric': None, 'log': 'Experiments/application_1637773802226_0043_4/output.log'})"
     ]
    }
   ],
   "source": [
    "from hops import experiment\n",
    "\n",
    "#Export from local paths\n",
    "experiment.launch(export_tf)\n",
    "\n",
    "#Export from hdfs paths\n",
    "experiment.launch(export_py)\n",
    "experiment.launch(export_sklearn)\n",
    "experiment.launch(export_torch)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "python",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 3
   },
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
